import csv
import random
from datetime import datetime
from time import strftime
from dbfpy import dbf
import numpy as np

'''
Find the first three appeared significant regions, check if they are overlaped with the designed risk areas.

calculate the false nagetive, false positive
'''

def getCurTime():
    """
    get current time
    Return value of the date string format(%Y-%m-%d %H:%M:%S)
    """
    format='%Y-%m-%d %H:%M:%S'
    sdate = None
    cdate = datetime.now()
    try:
        sdate = cdate.strftime(format)
    except:
        raise ValueError
    return sdate

def build_data_list(inputCSV):
    sKey = []
    fn = inputCSV
    ra = csv.DictReader(file(fn), dialect="excel")
    
    for record in ra:
        #print record[ra.fieldnames[0]], type(record[ra.fieldnames[-1]])
        for item in ra.fieldnames:
            temp = float(record[item])
            sKey.append(temp)
    sKey = np.array(sKey)
    sKey.shape=(-1,len(ra.fieldnames))
    return sKey

def build_pvalue_list(inputCSV):
    sKey = []
    fn = inputCSV
    ra = csv.DictReader(file(fn), dialect="excel")
    
    for record in ra:
        #print record[ra.fieldnames[0]], type(record[ra.fieldnames[-1]])
        #for item in ra.fieldnames:
        temp = float(record[ra.fieldnames[-1]])
        sKey.append(temp)
    sKey = np.array(sKey)
    #sKey.shape=(-1,len(ra.fieldnames))
    return sKey

def findRegionID(unitMatrix, clusterID):
    # unitMatrix: [unitID, regionID, P-value]
    regionID = []
    for item in unitMatrix:
        if int(item[0]) in clusterID:
            regionID.append(int(item[1]))
            #print item[0]
    regionID = np.unique(regionID)
    #regionID = [regionID]
    return regionID

def findUnitIDforRegionID(unitMatrix, regionID):
    # unitMatrix: [unitID, regionID, P-value]
    listID = []
    for item in unitMatrix:
        if int(item[1]) in regionID:
            listID.append(int(item[0]))
    return listID

def calSum(data):
    # sum: [mixed, rural, urban, total]
    sum = [0, 0, 0, 0]
    for item in mixed:
        sum[0] += data[int(item)]
    for item in rural:
        sum[1] += data[int(item)]
    for item in urban:
        sum[2] += data[int(item)]
    sum[3] = sum[0] + sum[1] + sum[2]
    return sum

def calTPFPFN(dataMatrix, usedClusterID, unitID, pop, cancer):
    temp_result = np.zeros(12)
    #[POP_TP, POP_FP, POP_FN, DIS_TP, DIS_FP, DIS_FN, COUNT_TP, COUNT_FP, COUNT_FN,
    #   POP_TP/(FP+FN), DIS_TP/(FP+FN), COUNT_TP/(FP+FN)]
    for item in unitID:
        if int(item) in usedClusterID:
            temp_result[0] += dataMatrix[int(item), 1]
            temp_result[3] += dataMatrix[int(item), repeat + 2]
            temp_result[6] += 1
        else:
            temp_result[1] += dataMatrix[int(item), 1]
            temp_result[4] += dataMatrix[int(item), repeat + 2]
            temp_result[7] += 1
                
    temp_result[2] = pop - temp_result[0]
    temp_result[5] = cancer - temp_result[3]
    temp_result[8] = len(usedClusterID) - temp_result[6]

    temp_result[9] = temp_result[0]/(temp_result[0] + temp_result[1] + temp_result[2])
    temp_result[10] = temp_result[3]/(temp_result[3] + temp_result[4] + temp_result[5])
    temp_result[11] = temp_result[6]/(temp_result[6] + temp_result[7] + temp_result[8])
    return temp_result

def findClusterID(pvalue):
    # pvalue [id, reg1, reg2, ... min]
    # region [id, reg1, reg2, ...]
    pLen = pvalue.shape
    clusterID = []
    cluster1 = []
    cluster2 = []
    cluster3 = []
    
    cluster1, nextID = findCluster(clusterID, pvalue, 2)
    #print cluster1, nextID
    for item in cluster1:
        clusterID.append(int(item))
    
    if (nextID < pLen[1] - 1) and (nextID > 0):
        cluster2, nextID = findCluster(clusterID, pvalue, nextID)
    #print nextID
    if (nextID < pLen[1] - 1) and (nextID > 0):
        for item in cluster2:
            clusterID.append(int(item))
        cluster3, nextID = findCluster(clusterID, pvalue, nextID)
    #print clusterID
    return cluster1, cluster2, cluster3


def findCluster(clusterID, pvalue, start):
    # pvalue [id, reg1, reg2, ... min]
    pLen = pvalue.shape
    findClusterID = []
    
    for i in range(start, pLen[1]-1):
        templist = []
        for item in pvalue:
            if item[i] < significanceLevel:
                templist.append(int(item[0]))
        for item in templist:
            if item not in clusterID:
                findClusterID.append(item)
        if len(findClusterID) > 0:
            return findClusterID, i+1
            break
    return findClusterID, -1

def calSeperateTPFPFN(data, usedClusterID, unitID, pop):
    temp_result = [0, 0, 0, 0]
    #[TP, FP, FN, TP/(TP+FP+FN)]
    for item in unitID:
        if int(item) in usedClusterID:
            temp_result[0] += data[int(item)]
        else:
            temp_result[1] += data[int(item)]
                
    temp_result[2] = pop - temp_result[0]

    temp_result[3] = temp_result[0]/(temp_result[0] + temp_result[1] + temp_result[2])

    return temp_result

def findHighestScore(listID):
    win_score = [0, 0, 0, 0]
    win_p = -1
    #print repeat
    
    for p in range(0,3):
        if p == 0:
            usedClusterID = mixed
        elif p == 1:
            usedClusterID = rural
        elif p == 2:
            usedClusterID = urban
        if type == 0:
            temp = calSeperateTPFPFN(dataMatrix[:,1], usedClusterID, listID, riskarea_pop[p])
        elif type == 1:
            temp = calSeperateTPFPFN(dataMatrix[:, repeat + 2], usedClusterID, listID, riskarea_dis[p])
        elif type == 2:
            temp = calSeperateTPFPFN(dataCount, usedClusterID, listID, len(usedClusterID))

        if temp[-1] > win_score[-1]:
            win_score = temp
            win_p = p
    return win_score, win_p


#--------------------------------------------------------------------------
#MAIN
if __name__ == "__main__":
    print "begin at " + getCurTime()
    mixed = [91,98,101,104,114,115,119,126,131,142,146,147,154,162,168,172]
    rural = [8,9,10,11,12,13,14,15,17,19,20,26,28,33,34,37]
    urban = [105,107,112,120,122,125,127,128,130,133,134,141,143,149,152,155]

    hot_1 = [9,130,147]
    hot_2 = hot_1 + [10,133,154]
    hot_4 = hot_2 + [12,17,125,131,141,146]
    hot_8 = hot_4 + [14,19,20,26,114,115,119,120,128,134,149,168]
    hot_16 = mixed + rural + urban
    


    dataused = hot_16
    if len(dataused)/3 < 10:
        unitCSV = 'C:/_DATA/CancerData/SatScan/mult6000/three0' + str(len(dataused)/3) + '_format.csv'
    else:
        unitCSV = 'C:/_DATA/CancerData/SatScan/mult6000/three' + str(len(dataused)/3) + '_format.csv'
        
    dataMatrix = build_data_list(unitCSV)  # [id, pop, cancer1, cancer2, cancer3]
    riskarea_pop = calSum(dataMatrix[:,1])  # [mixed, rural, urban, total]
    
    type = 0 # 0: pop, 1: cancer, 2: count
    saveOutput = []
    for significanceLevel in range(1, 20):
        significanceLevel = 0.05 * significanceLevel

        dataCount = np.ones(len(dataMatrix))
        power = 0
        power2 = 0
        power3 = 0
        output = np.array([])
        scoreID = np.zeros((1000,3))
        for i in range(0,1000):
            for j in range(0,3):
                scoreID[i, j] = -1
        #print scoreID
        row = 0
        count = 1
        for repeat in range(0,1000):
            #print repeat
            #row += 1
            #if int(row/100) > 0:
                #print count*100
                #row = 0
                #count += 1
            #repeat += 450
            riskarea_dis = calSum(dataMatrix[:, repeat + 2])    #[mixed, rural, urban, total]
            #for item in dataused:
                #riskarea_dis += dataMatrix[int(item), repeat + 2]
            '''
            
            if len(dataused)/3 < 10:
                pvalueCSV = 'C:/_DATA/CancerData/SatScan/mult6000/redcap/three0'
                pvalueCSV += str(len(dataused)/3) + '/LLR/EBS/constraint/5p_improved/LLR_EBS_high_' + str(repeat) + '_pvalue.csv'
                #regionCSV = pvalueCSV[:-11] + '.csv'
            else:
                pvalueCSV = 'C:/_DATA/CancerData/SatScan/mult6000/redcap/three'
                pvalueCSV += str(len(dataused)/3) + '/LLR/EBS/constraint/5p_improved/LLR_EBS_high_' + str(repeat) + '_pvalue.csv'
                #regionCSV = pvalueCSV[:-11] + '.csv'
            '''
            pvalueCSV = 'C:/_DATA/CancerData/SatScan/mult6000/redcap/three16/'
            pvalueCSV += 'CLK/EBS/constraint/_Full-Order-CLK_SO' + str(repeat) + '_pvalue.csv'
            #pvalueCSV += 'WARD/EBS/constraint/_WARD_SO' + str(repeat) + '_pvalue.csv'
            #pvalueCSV += 'ALK/newCTG/EBS/3/_Full-Order-ALK_SO' + str(repeat) + '_pvalue.csv'
            #pvalueCSV += 'LLR/EBS/constraint/LLR_EBS_high_' + str(repeat) + '_pvalue.csv'
            #pvalueCSV += 'LLR/EBS/constraint/5p_improved/LLR_EBS_high_' + str(repeat) + '_pvalue.csv'
            pvalue = build_data_list(pvalueCSV)
            #region = build_data_list(regionCSV)
            #print region

            cluster1, cluster2, cluster3 = findClusterID(pvalue)
            #print cluster1, cluster2, cluster3
            flag = 0
            #print cluster1, cluster2, cluster3
            detectedArea = [-1, -1, -1]
            total_result = np.zeros(4)
            for p in range(0,3):
                if p == 0:
                    usedClusterID = cluster1
                elif p == 1:
                    usedClusterID = cluster2
                elif p == 2:
                    usedClusterID = cluster3
                    
                if len(usedClusterID) > 0:
                    flag = 1
                    power += 1
                    temp_result, scoreID[repeat, p] = findHighestScore(usedClusterID)
                    #print temp_result, scoreID[repeat]
                    if scoreID[repeat, p] > -1:
                        power2 += 1
                        if scoreID[repeat, p] not in detectedArea:
                            detectedArea[int(scoreID[repeat, p])] = scoreID[repeat, p]
                            power3 += 1
                    for i in range(0,3):
                        total_result[i] += temp_result[i]
            temp_value = total_result[0] + total_result[1] + total_result[2]

            if (scoreID[repeat, 0] > -1) or (scoreID[repeat, 1] > -1) or (scoreID[repeat, 2] > -1):
                total_result[3] = total_result[0]/(total_result[0] + total_result[1] + total_result[2])
                #if int(total_result[0] + total_result[1] + total_result[2]) == 0:
                    #print repeat, 'error'
                    #break
                output = np.append(output, total_result)
        #print output
        filePath = 'C:/_DATA/CancerData/SatScan/mult6000/LLR_ebs' + str(len(dataused)/3) + '_original_newCTG_' + str(significanceLevel) + '_'
        output.shape = (-1, 4)
        #print output
        if type == 0:
            filePath += 'pop.csv'
        if type == 1:
            filePath += 'cancer.csv'
        if type == 2:
            filePath += 'count.csv'
        #np.savetxt(filePath, output, delimiter=',', fmt = '%10.5f')
        #np.savetxt(filePath[:-4] + 'scoreID.csv', scoreID, delimiter=',', fmt = '%10.5f')
        print '------------------------'
        print significanceLevel
        print np.average(output[:,3])
    ##        # generate [unitID, regionID, p_value]
    ##        unitAttri = []    # all units
    ##        unitPvalue = []   # units with p_value < 0.05
    ##        i = 0
    ##        for item in pvalue:
    ##            unitAttri.append([i, region[i], item])
    ##            if item < 0.05:
    ##                unitPvalue.append([i, region[i], item])
    ##            i += 1
    ##
    ##        if len(unitPvalue) > 0:
    ##            unitAttri = np.array(unitAttri)
    ##            unitPvalue = np.array(unitPvalue)
    ##        else:
    ##            continue
    ##
    ##        total_result = np.zeros(12)
    ##        for p in range(0,3):
    ##            if p == 0:
    ##                usedClusterID = mixed
    ##            elif p == 1:
    ##                usedClusterID = rural
    ##            elif p == 2:
    ##                usedClusterID = urban
    ##            regionID = findRegionID(unitPvalue, usedClusterID)
    ##            #regionID = findRegionID(unitPvalue, rural)
    ##            #regionID = findRegionID(unitPvalue, urban)
    ##            #print mixedID, ruralID, urbanID
    ##            #print regionID
    ##            if len(regionID) > 0:
    ##                power += 1
    ##                listID = findUnitIDforRegionID(unitAttri, regionID)
    ##                #print listID
    ##            else:
    ##                continue
    ##            
    ##            temp_result = calTPFPFN(dataMatrix, usedClusterID, listID, riskarea_pop[p], riskarea_dis[p])
    ##            
    ##            for i in range(0,9):
    ##                total_result[i] += temp_result[i]
    ##            #print total_result
    ##
    ##            if p == 0:
    ##                mixed_output = np.append(mixed_output, temp_result)
    ##            elif p == 1:
    ##                rural_output = np.append(rural_output, temp_result)
    ##            elif p == 2:
    ##                urban_output = np.append(urban_output, temp_result)
    ##                
    ##        total_result[9] = total_result[0]/(total_result[0] + total_result[1] + total_result[2])
    ##        total_result[10] = total_result[3]/(total_result[3] + total_result[4] + total_result[5])
    ##        total_result[11] = total_result[6]/(total_result[6] + total_result[7] + temp_result[8])
    ##        output = np.append(output, total_result)
    ##    #print output
    ##    filePath = 'C:/_DATA/CancerData/SatScan/mult6000/so_result' + str(len(dataused)/3) + '_newCTG_ebs'
    ##    mixed_output.shape = (-1, 12)
    ##    np.savetxt(filePath + '_mixed.csv', mixed_output, delimiter=',', fmt = '%10.5f')
    ##    rural_output.shape = (-1, 12)
    ##    np.savetxt(filePath + '_rural.csv', rural_output, delimiter=',', fmt = '%10.5f')
    ##    urban_output.shape = (-1, 12)
    ##    np.savetxt(filePath + '_urban.csv', urban_output, delimiter=',', fmt = '%10.5f')
    ##    output.shape = (-1, 12)
    ##    np.savetxt(filePath + '_total.csv', output, delimiter=',', fmt = '%10.5f')

        
        print 'power = ' + str(power)
        print 'power2 = ', power2
        print 'power3 = ', power3
        saveOutput.append([np.average(output[:,3]), power, power2, power3])
    np.savetxt(pvalueCSV[:-10] + '_power'+ str(type) +'.csv', saveOutput, delimiter=',', fmt = '%10.5f')
    print "end at " + getCurTime()
    print "========================================================================"  